}
else
{
- list_for_each_entry ( page, &d->page_list, list )
+ page_list_for_each ( page, &d->page_list )
{
printk(" DomPage %p: caf=%08lx, taf=%" PRtype_info "\n",
_p(page_to_mfn(page)),
p2m_pod_dump_data(d);
}
- list_for_each_entry ( page, &d->xenpage_list, list )
+ page_list_for_each ( page, &d->xenpage_list )
{
printk(" XenPage %p: caf=%08lx, taf=%" PRtype_info "\n",
_p(page_to_mfn(page)),
INIT_LIST_HEAD(&d->arch.pdev_list);
d->arch.relmem = RELMEM_not_started;
- INIT_LIST_HEAD(&d->arch.relmem_list);
+ INIT_PAGE_LIST_HEAD(&d->arch.relmem_list);
pdpt_order = get_order_from_bytes(PDPT_L1_ENTRIES * sizeof(l1_pgentry_t));
d->arch.mm_perdomain_pt = alloc_xenheap_pages(pdpt_order, 0);
#endif
static int relinquish_memory(
- struct domain *d, struct list_head *list, unsigned long type)
+ struct domain *d, struct page_list_head *list, unsigned long type)
{
- struct list_head *ent;
- struct page_info *page;
+ struct page_info *page, *cur;
unsigned long x, y;
int ret = 0;
/* Use a recursive lock, as we may enter 'free_domheap_page'. */
spin_lock_recursive(&d->page_alloc_lock);
- ent = list->next;
- while ( ent != list )
+ page = page_list_first(list);
+ while ( !page_list_is_eol(page, list) )
{
- page = list_entry(ent, struct page_info, list);
-
/* Grab a reference to the page so it won't disappear from under us. */
if ( unlikely(!get_page(page, d)) )
{
/* Couldn't get a reference -- someone is freeing this page. */
- ent = ent->next;
- list_move_tail(&page->list, &d->arch.relmem_list);
+ cur = page;
+ page = page_list_next(page, list);
+ page_list_move_tail(cur, list, &d->arch.relmem_list);
continue;
}
}
/* Follow the list chain and /then/ potentially free the page. */
- ent = ent->next;
- list_move_tail(&page->list, &d->arch.relmem_list);
- put_page(page);
+ cur = page;
+ page = page_list_next(page, list);
+ page_list_move_tail(cur, list, &d->arch.relmem_list);
+ put_page(cur);
if ( hypercall_preempt_check() )
{
}
}
- list_splice_init(&d->arch.relmem_list, list);
+ page_list_splice_init(&d->arch.relmem_list, list);
out:
spin_unlock_recursive(&d->page_alloc_lock);
}
si->first_p2m_pfn = pfn;
si->nr_p2m_frames = d->tot_pages - count;
- list_for_each_entry ( page, &d->page_list, list )
+ page_list_for_each ( page, &d->page_list )
{
mfn = page_to_mfn(page);
if ( get_gpfn_from_mfn(mfn) >= count )
struct domain *d = rcu_lock_domain_by_id(domctl->domain);
unsigned long max_pfns = domctl->u.getmemlist.max_pfns;
uint64_t mfn;
- struct list_head *list_ent;
+ struct page_info *page;
ret = -EINVAL;
if ( d != NULL )
goto getmemlist_out;
}
- ret = 0;
- list_ent = d->page_list.next;
- for ( i = 0; (i < max_pfns) && (list_ent != &d->page_list); i++ )
+ ret = i = 0;
+ page_list_for_each(page, &d->page_list)
{
- mfn = page_to_mfn(list_entry(
- list_ent, struct page_info, list));
+ if ( i >= max_pfns )
+ break;
+ mfn = page_to_mfn(page);
if ( copy_to_guest_offset(domctl->u.getmemlist.buffer,
i, &mfn, 1) )
{
ret = -EFAULT;
break;
}
- list_ent = mfn_to_page(mfn)->list.next;
+ ++i;
}
spin_unlock(&d->page_alloc_lock);
#include <xen/config.h>
#include <xen/init.h>
#include <xen/lib.h>
+#include <xen/mm.h>
#include <xen/compat.h>
#include <xen/dmi.h>
#include <asm/e820.h>
-#include <asm/mm.h>
#include <asm/page.h>
/* opt_mem: Limit of physical RAM. Any RAM beyond this point is ignored. */
page->count_info |= PGC_allocated | 1;
if ( unlikely(d->xenheap_pages++ == 0) )
get_knownalive_domain(d);
- list_add_tail(&page->list, &d->xenpage_list);
+ page_list_add_tail(page, &d->xenpage_list);
}
spin_unlock(&d->page_alloc_lock);
/* Unlink from original owner. */
if ( !(memflags & MEMF_no_refcount) )
d->tot_pages--;
- list_del(&page->list);
+ page_list_del(page, &d->page_list);
spin_unlock(&d->page_alloc_lock);
return 0;
ASSERT(hap_locked_by_me(d));
- if ( unlikely(list_empty(&d->arch.paging.hap.freelist)) )
+ pg = page_list_remove_head(&d->arch.paging.hap.freelist);
+ if ( unlikely(!pg) )
return NULL;
- pg = list_entry(d->arch.paging.hap.freelist.next, struct page_info, list);
- list_del(&pg->list);
d->arch.paging.hap.free_pages--;
p = hap_map_domain_page(page_to_mfn(pg));
ASSERT(hap_locked_by_me(d));
d->arch.paging.hap.free_pages++;
- list_add_tail(&pg->list, &d->arch.paging.hap.freelist);
+ page_list_add_tail(pg, &d->arch.paging.hap.freelist);
}
static struct page_info *hap_alloc_p2m_page(struct domain *d)
}
d->arch.paging.hap.free_pages++;
d->arch.paging.hap.total_pages++;
- list_add_tail(&pg->list, &d->arch.paging.hap.freelist);
+ page_list_add_tail(pg, &d->arch.paging.hap.freelist);
}
else if ( d->arch.paging.hap.total_pages > pages )
{
/* Need to return memory to domheap */
- ASSERT(!list_empty(&d->arch.paging.hap.freelist));
- pg = list_entry(d->arch.paging.hap.freelist.next,
- struct page_info, list);
- list_del(&pg->list);
+ pg = page_list_remove_head(&d->arch.paging.hap.freelist);
+ ASSERT(pg);
d->arch.paging.hap.free_pages--;
d->arch.paging.hap.total_pages--;
pg->count_info = 0;
void hap_domain_init(struct domain *d)
{
hap_lock_init(d);
- INIT_LIST_HEAD(&d->arch.paging.hap.freelist);
+ INIT_PAGE_LIST_HEAD(&d->arch.paging.hap.freelist);
/* This domain will use HAP for log-dirty mode */
paging_log_dirty_init(d, hap_enable_log_dirty, hap_disable_log_dirty,
pg->count_info = 1;
pg->u.inuse.type_info = 1 | PGT_validated;
- list_add_tail(&pg->list, &d->arch.p2m->pages);
+ page_list_add_tail(pg, &d->arch.p2m->pages);
ept_entry->emt = 0;
ept_entry->igmt = 0;
struct page_info *pg = d->arch.p2m->alloc_page(d);
if ( pg == NULL )
return 0;
- list_add_tail(&pg->list, &d->arch.p2m->pages);
+ page_list_add_tail(pg, &d->arch.p2m->pages);
pg->u.inuse.type_info = type | 1 | PGT_validated;
pg->count_info = 1;
struct page_info *pg = d->arch.p2m->alloc_page(d);
if ( pg == NULL )
return 0;
- list_add_tail(&pg->list, &d->arch.p2m->pages);
+ page_list_add_tail(pg, &d->arch.p2m->pages);
pg->u.inuse.type_info = PGT_l1_page_table | 1 | PGT_validated;
pg->count_info = 1;
for(i=0; i < 1 << order ; i++)
{
p = page + i;
- list_del(&p->list);
+ page_list_del(p, &d->page_list);
}
/* Then add the first one to the appropriate populate-on-demand list */
switch(order)
{
case 9:
- list_add_tail(&page->list, &p2md->pod.super); /* lock: page_alloc */
+ page_list_add_tail(page, &p2md->pod.super); /* lock: page_alloc */
p2md->pod.count += 1 << order;
break;
case 0:
- list_add_tail(&page->list, &p2md->pod.single); /* lock: page_alloc */
+ page_list_add_tail(page, &p2md->pod.single); /* lock: page_alloc */
p2md->pod.count += 1 ;
break;
default:
struct page_info *p = NULL;
int i;
- if ( order == 9 && list_empty(&p2md->pod.super) )
+ if ( order == 9 && page_list_empty(&p2md->pod.super) )
{
return NULL;
}
- else if ( order == 0 && list_empty(&p2md->pod.single) )
+ else if ( order == 0 && page_list_empty(&p2md->pod.single) )
{
unsigned long mfn;
struct page_info *q;
- BUG_ON( list_empty(&p2md->pod.super) );
+ BUG_ON( page_list_empty(&p2md->pod.super) );
/* Break up a superpage to make single pages. NB count doesn't
* need to be adjusted. */
printk("%s: Breaking up superpage.\n", __func__);
- p = list_entry(p2md->pod.super.next, struct page_info, list);
- list_del(&p->list);
+ p = page_list_remove_head(&p2md->pod.super);
mfn = mfn_x(page_to_mfn(p));
for ( i=0; i<(1<<9); i++ )
{
q = mfn_to_page(_mfn(mfn+i));
- list_add_tail(&q->list, &p2md->pod.single);
+ page_list_add_tail(q, &p2md->pod.single);
}
}
switch ( order )
{
case 9:
- BUG_ON( list_empty(&p2md->pod.super) );
- p = list_entry(p2md->pod.super.next, struct page_info, list);
+ BUG_ON( page_list_empty(&p2md->pod.super) );
+ p = page_list_remove_head(&p2md->pod.super);
p2md->pod.count -= 1 << order; /* Lock: page_alloc */
break;
case 0:
- BUG_ON( list_empty(&p2md->pod.single) );
- p = list_entry(p2md->pod.single.next, struct page_info, list);
+ BUG_ON( page_list_empty(&p2md->pod.single) );
+ p = page_list_remove_head(&p2md->pod.single);
p2md->pod.count -= 1;
break;
default:
BUG();
}
- list_del(&p->list);
-
/* Put the pages back on the domain page_list */
for ( i = 0 ; i < (1 << order) ; i++ )
{
BUG_ON(page_get_owner(p + i) != d);
- list_add_tail(&p[i].list, &d->page_list);
+ page_list_add_tail(p + i, &d->page_list);
}
return p;
spin_lock(&d->page_alloc_lock);
if ( (p2md->pod.count - pod_target) > (1>>9)
- && !list_empty(&p2md->pod.super) )
+ && !page_list_empty(&p2md->pod.super) )
order = 9;
else
order = 0;
p2m_pod_empty_cache(struct domain *d)
{
struct p2m_domain *p2md = d->arch.p2m;
- struct list_head *q, *p;
+ struct page_info *page;
spin_lock(&d->page_alloc_lock);
- list_for_each_safe(p, q, &p2md->pod.super) /* lock: page_alloc */
+ while ( (page = page_list_remove_head(&p2md->pod.super)) )
{
int i;
- struct page_info *page;
- list_del(p);
-
- page = list_entry(p, struct page_info, list);
-
for ( i = 0 ; i < (1 << 9) ; i++ )
{
BUG_ON(page_get_owner(page + i) != d);
- list_add_tail(&page[i].list, &d->page_list);
+ page_list_add_tail(page + i, &d->page_list);
}
p2md->pod.count -= 1<<9;
}
- list_for_each_safe(p, q, &p2md->pod.single)
+ while ( (page = page_list_remove_head(&p2md->pod.single)) )
{
- struct page_info *page;
-
- list_del(p);
-
- page = list_entry(p, struct page_info, list);
-
BUG_ON(page_get_owner(page) != d);
- list_add_tail(&page->list, &d->page_list);
+ page_list_add_tail(page, &d->page_list);
p2md->pod.count -= 1;
}
* NB that this is a zero-sum game; we're increasing our cache size
* by increasing our 'debt'. Since we hold the p2m lock,
* (entry_count - count) must remain the same. */
- if ( !list_empty(&p2md->pod.super) && i < limit )
+ if ( !page_list_empty(&p2md->pod.super) && i < limit )
break;
}
}
/* If we're low, start a sweep */
- if ( order == 9 && list_empty(&p2md->pod.super) )
+ if ( order == 9 && page_list_empty(&p2md->pod.super) )
p2m_pod_emergency_sweep_super(d);
- if ( list_empty(&p2md->pod.single) &&
+ if ( page_list_empty(&p2md->pod.single) &&
( ( order == 0 )
- || (order == 9 && list_empty(&p2md->pod.super) ) ) )
+ || (order == 9 && page_list_empty(&p2md->pod.super) ) ) )
p2m_pod_emergency_sweep(d);
/* Keep track of the highest gfn demand-populated by a guest fault */
memset(p2m, 0, sizeof(*p2m));
p2m_lock_init(p2m);
- INIT_LIST_HEAD(&p2m->pages);
- INIT_LIST_HEAD(&p2m->pod.super);
- INIT_LIST_HEAD(&p2m->pod.single);
+ INIT_PAGE_LIST_HEAD(&p2m->pages);
+ INIT_PAGE_LIST_HEAD(&p2m->pod.super);
+ INIT_PAGE_LIST_HEAD(&p2m->pod.single);
p2m->set_entry = p2m_set_entry;
p2m->get_entry = p2m_gfn_to_mfn;
{
mfn_t mfn = _mfn(INVALID_MFN);
- struct list_head *entry;
struct page_info *page, *p2m_top;
unsigned int page_count = 0;
unsigned long gfn = -1UL;
p2m_unlock(p2m);
return -ENOMEM;
}
- list_add_tail(&p2m_top->list, &p2m->pages);
+ page_list_add_tail(p2m_top, &p2m->pages);
p2m_top->count_info = 1;
p2m_top->u.inuse.type_info =
goto error;
/* Copy all existing mappings from the page list and m2p */
- for ( entry = d->page_list.next;
- entry != &d->page_list;
- entry = entry->next )
+ page_list_for_each(page, &d->page_list)
{
- page = list_entry(entry, struct page_info, list);
mfn = page_to_mfn(page);
gfn = get_gpfn_from_mfn(mfn_x(mfn));
page_count++;
/* Return all the p2m pages to Xen.
* We know we don't have any extra mappings to these pages */
{
- struct list_head *entry, *n;
struct page_info *pg;
struct p2m_domain *p2m = d->arch.p2m;
p2m_lock(p2m);
d->arch.phys_table = pagetable_null();
- list_for_each_safe(entry, n, &p2m->pages)
- {
- pg = list_entry(entry, struct page_info, list);
- list_del(entry);
+ while ( (pg = page_list_remove_head(&p2m->pages)) )
p2m->free_page(d, pg);
- }
p2m_unlock(p2m);
}
shadow_lock_init(d);
for ( i = 0; i <= SHADOW_MAX_ORDER; i++ )
INIT_LIST_HEAD(&d->arch.paging.shadow.freelists[i]);
- INIT_LIST_HEAD(&d->arch.paging.shadow.p2m_freelist);
+ INIT_PAGE_LIST_HEAD(&d->arch.paging.shadow.p2m_freelist);
INIT_LIST_HEAD(&d->arch.paging.shadow.pinned_shadows);
/* Use shadow pagetables for log-dirty support */
*/
page_set_owner(&pg[i], d);
pg[i].count_info = 1;
- list_add_tail(&pg[i].list, &d->arch.paging.shadow.p2m_freelist);
+ page_list_add_tail(&pg[i], &d->arch.paging.shadow.p2m_freelist);
}
return 1;
}
static struct page_info *
shadow_alloc_p2m_page(struct domain *d)
{
- struct list_head *entry;
struct page_info *pg;
mfn_t mfn;
void *p;
shadow_lock(d);
- if ( list_empty(&d->arch.paging.shadow.p2m_freelist) &&
+ if ( page_list_empty(&d->arch.paging.shadow.p2m_freelist) &&
!sh_alloc_p2m_pages(d) )
{
shadow_unlock(d);
return NULL;
}
- entry = d->arch.paging.shadow.p2m_freelist.next;
- list_del(entry);
+ pg = page_list_remove_head(&d->arch.paging.shadow.p2m_freelist);
shadow_unlock(d);
- pg = list_entry(entry, struct page_info, list);
mfn = page_to_mfn(pg);
p = sh_map_domain_page(mfn);
clear_page(p);
{
struct vcpu *v;
mfn_t mfn;
- struct list_head *entry, *n;
struct page_info *pg;
ASSERT(d->is_dying);
}
#endif /* (SHADOW_OPTIMIZATIONS & (SHOPT_VIRTUAL_TLB|SHOPT_OUT_OF_SYNC)) */
- list_for_each_safe(entry, n, &d->arch.paging.shadow.p2m_freelist)
- {
- list_del(entry);
- pg = list_entry(entry, struct page_info, list);
+ while ( (pg = page_list_remove_head(&d->arch.paging.shadow.p2m_freelist)) )
shadow_free_p2m_page(d, pg);
- }
if ( d->arch.paging.shadow.total_pages != 0 )
{
for_each_online_node(i)
page_num_node[i] = 0;
- list_for_each_entry(page, &d->page_list, list)
+ page_list_for_each(page, &d->page_list)
{
i = phys_to_nid(page_to_mfn(page) << PAGE_SHIFT);
page_num_node[i]++;
spin_lock_init(&d->page_alloc_lock);
spin_lock_init(&d->shutdown_lock);
spin_lock_init(&d->hypercall_deadlock_mutex);
- INIT_LIST_HEAD(&d->page_list);
- INIT_LIST_HEAD(&d->xenpage_list);
+ INIT_PAGE_LIST_HEAD(&d->page_list);
+ INIT_PAGE_LIST_HEAD(&d->xenpage_list);
if ( domcr_flags & DOMCRF_hvm )
d->is_hvm = 1;
/* Okay, add the page to 'e'. */
if ( unlikely(e->tot_pages++ == 0) )
get_knownalive_domain(e);
- list_add_tail(&page->list, &e->page_list);
+ page_list_add_tail(page, &e->page_list);
page_set_owner(page, e);
spin_unlock(&e->page_alloc_lock);
static long memory_exchange(XEN_GUEST_HANDLE(xen_memory_exchange_t) arg)
{
struct xen_memory_exchange exch;
- LIST_HEAD(in_chunk_list);
- LIST_HEAD(out_chunk_list);
+ PAGE_LIST_HEAD(in_chunk_list);
+ PAGE_LIST_HEAD(out_chunk_list);
unsigned long in_chunk_order, out_chunk_order;
xen_pfn_t gpfn, gmfn, mfn;
unsigned long i, j, k;
goto fail;
}
- list_add(&page->list, &in_chunk_list);
+ page_list_add(page, &in_chunk_list);
}
}
goto fail;
}
- list_add(&page->list, &out_chunk_list);
+ page_list_add(page, &out_chunk_list);
}
/*
*/
/* Destroy final reference to each input page. */
- while ( !list_empty(&in_chunk_list) )
+ while ( (page = page_list_remove_head(&in_chunk_list)) )
{
- page = list_entry(in_chunk_list.next, struct page_info, list);
- list_del(&page->list);
if ( !test_and_clear_bit(_PGC_allocated, &page->count_info) )
BUG();
mfn = page_to_mfn(page);
/* Assign each output page to the domain. */
j = 0;
- while ( !list_empty(&out_chunk_list) )
+ while ( (page = page_list_remove_head(&out_chunk_list)) )
{
- page = list_entry(out_chunk_list.next, struct page_info, list);
- list_del(&page->list);
if ( assign_pages(d, page, exch.out.extent_order,
MEMF_no_refcount) )
BUG();
*/
fail:
/* Reassign any input pages we managed to steal. */
- while ( !list_empty(&in_chunk_list) )
- {
- page = list_entry(in_chunk_list.next, struct page_info, list);
- list_del(&page->list);
+ while ( (page = page_list_remove_head(&in_chunk_list)) )
if ( assign_pages(d, page, 0, MEMF_no_refcount) )
BUG();
- }
/* Free any output pages we managed to allocate. */
- while ( !list_empty(&out_chunk_list) )
- {
- page = list_entry(out_chunk_list.next, struct page_info, list);
- list_del(&page->list);
+ while ( (page = page_list_remove_head(&out_chunk_list)) )
free_domheap_pages(page, exch.out.extent_order);
- }
exch.nr_exchanged = i << in_chunk_order;
#endif
static DEFINE_SPINLOCK(page_scrub_lock);
-LIST_HEAD(page_scrub_list);
+PAGE_LIST_HEAD(page_scrub_list);
static unsigned long scrub_pages;
/*********************
#define page_to_zone(pg) (is_xen_heap_page(pg) ? MEMZONE_XEN : \
(fls(page_to_mfn(pg)) - 1))
-typedef struct list_head heap_by_zone_and_order_t[NR_ZONES][MAX_ORDER+1];
+typedef struct page_list_head heap_by_zone_and_order_t[NR_ZONES][MAX_ORDER+1];
static heap_by_zone_and_order_t *_heap[MAX_NUMNODES];
#define heap(node, zone, order) ((*_heap[node])[zone][order])
for ( i = 0; i < NR_ZONES; i++ )
for ( j = 0; j <= MAX_ORDER; j++ )
- INIT_LIST_HEAD(&(*_heap[node])[i][j]);
+ INIT_PAGE_LIST_HEAD(&(*_heap[node])[i][j]);
}
/* Allocate 2^@order contiguous pages. */
/* Find smallest order which can satisfy the request. */
for ( j = order; j <= MAX_ORDER; j++ )
- if ( !list_empty(&heap(node, zone, j)) )
+ if ( (pg = page_list_remove_head(&heap(node, zone, j))) )
goto found;
} while ( zone-- > zone_lo ); /* careful: unsigned zone may wrap */
return NULL;
found:
- pg = list_entry(heap(node, zone, j).next, struct page_info, list);
- list_del(&pg->list);
-
/* We may have to halve the chunk a number of times. */
while ( j != order )
{
PFN_ORDER(pg) = --j;
- list_add_tail(&pg->list, &heap(node, zone, j));
+ page_list_add_tail(pg, &heap(node, zone, j));
pg += 1 << j;
}
if ( allocated_in_map(page_to_mfn(pg)-mask) ||
(PFN_ORDER(pg-mask) != order) )
break;
- list_del(&(pg-mask)->list);
pg -= mask;
+ page_list_del(pg, &heap(node, zone, order));
}
else
{
if ( allocated_in_map(page_to_mfn(pg)+mask) ||
(PFN_ORDER(pg+mask) != order) )
break;
- list_del(&(pg+mask)->list);
+ page_list_del(pg + mask, &heap(node, zone, order));
}
order++;
}
PFN_ORDER(pg) = order;
- list_add_tail(&pg->list, &heap(node, zone, order));
+ page_list_add_tail(pg, &heap(node, zone, order));
spin_unlock(&heap_lock);
}
page_set_owner(&pg[i], d);
wmb(); /* Domain pointer must be visible before updating refcnt. */
pg[i].count_info = PGC_allocated | 1;
- list_add_tail(&pg[i].list, &d->page_list);
+ page_list_add_tail(&pg[i], &d->page_list);
}
spin_unlock(&d->page_alloc_lock);
spin_lock_recursive(&d->page_alloc_lock);
for ( i = 0; i < (1 << order); i++ )
- list_del(&pg[i].list);
+ page_list_del2(&pg[i], &d->xenpage_list, &d->arch.relmem_list);
d->xenheap_pages -= 1 << order;
drop_dom_ref = (d->xenheap_pages == 0);
for ( i = 0; i < (1 << order); i++ )
{
BUG_ON((pg[i].u.inuse.type_info & PGT_count_mask) != 0);
- list_del(&pg[i].list);
+ page_list_del2(&pg[i], &d->page_list, &d->arch.relmem_list);
}
d->tot_pages -= 1 << order;
{
page_set_owner(&pg[i], NULL);
spin_lock(&page_scrub_lock);
- list_add(&pg[i].list, &page_scrub_list);
+ page_list_add(&pg[i], &page_scrub_list);
scrub_pages++;
spin_unlock(&page_scrub_lock);
}
static void page_scrub_softirq(void)
{
- struct list_head *ent;
+ PAGE_LIST_HEAD(list);
struct page_info *pg;
void *p;
int i;
do {
spin_lock(&page_scrub_lock);
- if ( unlikely((ent = page_scrub_list.next) == &page_scrub_list) )
- {
- spin_unlock(&page_scrub_lock);
- goto out;
- }
-
/* Peel up to 16 pages from the list. */
for ( i = 0; i < 16; i++ )
{
- if ( ent->next == &page_scrub_list )
+ if ( !(pg = page_list_remove_head(&page_scrub_list)) )
break;
- ent = ent->next;
+ page_list_add_tail(pg, &list);
}
- /* Remove peeled pages from the list. */
- ent->next->prev = &page_scrub_list;
- page_scrub_list.next = ent->next;
- scrub_pages -= (i+1);
+ if ( unlikely(i == 0) )
+ {
+ spin_unlock(&page_scrub_lock);
+ goto out;
+ }
+
+ scrub_pages -= i;
spin_unlock(&page_scrub_lock);
- /* Working backwards, scrub each page in turn. */
- while ( ent != &page_scrub_list )
- {
- pg = list_entry(ent, struct page_info, list);
- ent = ent->prev;
+ /* Scrub each page in turn. */
+ while ( (pg = page_list_remove_head(&list)) ) {
p = map_domain_page(page_to_mfn(pg));
scrub_page(p);
unmap_domain_page(p);
{
unsigned long mfn, gfn, flags;
u64 iommu_l2e;
- struct list_head *entry;
struct page_info *page;
struct hvm_iommu *hd;
int iw = IOMMU_IO_WRITE_ENABLED;
if ( hd->p2m_synchronized )
goto out;
- for ( entry = d->page_list.next; entry != &d->page_list;
- entry = entry->next )
+ page_list_for_each ( page, &d->page_list )
{
- page = list_entry(entry, struct page_info, list);
mfn = page_to_mfn(page);
gfn = get_gpfn_from_mfn(mfn);
#include <xen/pci_regs.h>
#include <asm/amd-iommu.h>
#include <asm/hvm/svm/amd-iommu-proto.h>
-#include <asm/mm.h>
extern unsigned short ivrs_bdf_entries;
extern struct ivrs_mappings *ivrs_mappings;
spin_lock(&d->page_alloc_lock);
- list_for_each_entry ( page, &d->page_list, list )
+ page_list_for_each ( page, &d->page_list )
{
if ( (page->u.inuse.type_info & PGT_type_mask) == PGT_writable_page )
{
/* Memory allocation */
struct list_head freelists[SHADOW_MAX_ORDER + 1];
- struct list_head p2m_freelist;
+ struct page_list_head p2m_freelist;
unsigned int total_pages; /* number of pages allocated */
unsigned int free_pages; /* number of pages on freelists */
unsigned int p2m_pages; /* number of pages allocates to p2m */
int locker;
const char *locker_function;
- struct list_head freelist;
+ struct page_list_head freelist;
unsigned int total_pages; /* number of pages allocated */
unsigned int free_pages; /* number of pages on freelists */
unsigned int p2m_pages; /* number of pages allocates to p2m */
RELMEM_l2,
RELMEM_done,
} relmem;
- struct list_head relmem_list;
+ struct page_list_head relmem_list;
cpuid_input_t cpuids[MAX_CPUID_INPUT];
} __cacheline_aligned;
* Per-page-frame information.
*
* Every architecture must ensure the following:
- * 1. 'struct page_info' contains a 'struct list_head list'.
+ * 1. 'struct page_info' contains a 'struct page_list_entry list'.
* 2. Provide a PFN_ORDER() macro for accessing the order of a free page.
*/
#define PFN_ORDER(_pfn) ((_pfn)->u.free.order)
+#ifndef __i386__
+# undef page_list_entry
+struct page_list_entry
+{
+ unsigned int next, prev;
+ unsigned long _pad_for_sh_; /* until struct shadow_page_info gets updated */
+};
+#endif
+
struct page_info
{
/* Each frame can be threaded onto a doubly-linked list. */
- struct list_head list;
+ struct page_list_entry list;
/* Reference count and various PGC_xxx flags and fields. */
unsigned long count_info;
const char *locker_function; /* Func that took it */
/* Pages used to construct the p2m */
- struct list_head pages;
+ struct page_list_head pages;
/* Functions to call to get or free pages for the p2m */
struct page_info * (*alloc_page )(struct domain *d);
* protect moving stuff from the PoD cache to the domain page list.
*/
struct {
- struct list_head super, /* List of superpages */
+ struct page_list_head super, /* List of superpages */
single; /* Non-super lists */
int count, /* # of pages in cache lists */
entry_count; /* # of pages in p2m marked pod */
#ifndef __ASM_PERFC_H__
#define __ASM_PERFC_H__
-#include <asm/mm.h>
static inline void arch_perfc_printall(void)
{
#define MAX_ORDER 20 /* 2^20 contiguous pages */
#endif
+#define page_list_entry list_head
+
+#include <asm/mm.h>
+
+#ifndef page_list_entry
+struct page_list_head
+{
+ struct page_info *next, *tail;
+};
+/* These must only have instances in struct page_info. */
+# define page_list_entry
+
+# define PAGE_LIST_HEAD_INIT(name) { NULL, NULL }
+# define PAGE_LIST_HEAD(name) \
+ struct page_list_head name = PAGE_LIST_HEAD_INIT(name)
+# define INIT_PAGE_LIST_HEAD(head) ((head)->tail = (head)->next = NULL)
+# define INIT_PAGE_LIST_ENTRY(ent) ((ent)->prev = (ent)->next = ~0)
+
+static inline int
+page_list_empty(const struct page_list_head *head)
+{
+ return !head->next;
+}
+static inline struct page_info *
+page_list_first(const struct page_list_head *head)
+{
+ return head->next;
+}
+static inline struct page_info *
+page_list_next(const struct page_info *page,
+ const struct page_list_head *head)
+{
+ return page != head->tail ? mfn_to_page(page->list.next) : NULL;
+}
+static inline struct page_info *
+page_list_prev(const struct page_info *page,
+ const struct page_list_head *head)
+{
+ return page != head->next ? mfn_to_page(page->list.prev) : NULL;
+}
+static inline int
+page_list_is_eol(const struct page_info *page,
+ const struct page_list_head *head)
+{
+ return !page;
+}
+static inline void
+page_list_add(struct page_info *page, struct page_list_head *head)
+{
+ if ( head->next )
+ {
+ page->list.next = page_to_mfn(head->next);
+ head->next->list.prev = page_to_mfn(page);
+ }
+ else
+ {
+ head->tail = page;
+ page->list.next = ~0;
+ }
+ page->list.prev = ~0;
+ head->next = page;
+}
+static inline void
+page_list_add_tail(struct page_info *page, struct page_list_head *head)
+{
+ page->list.next = ~0;
+ if ( head->next )
+ {
+ page->list.prev = page_to_mfn(head->tail);
+ head->tail->list.next = page_to_mfn(page);
+ }
+ else
+ {
+ page->list.prev = ~0;
+ head->next = page;
+ }
+ head->tail = page;
+}
+static inline bool_t
+__page_list_del_head(struct page_info *page, struct page_list_head *head,
+ struct page_info *next, struct page_info *prev)
+{
+ if ( head->next == page )
+ {
+ if ( head->tail != page )
+ {
+ next->list.prev = ~0;
+ head->next = next;
+ }
+ else
+ head->tail = head->next = NULL;
+ return 1;
+ }
+
+ if ( head->tail == page )
+ {
+ prev->list.next = ~0;
+ head->tail = prev;
+ return 1;
+ }
+
+ return 0;
+}
+static inline void
+page_list_del(struct page_info *page, struct page_list_head *head)
+{
+ struct page_info *next = mfn_to_page(page->list.next);
+ struct page_info *prev = mfn_to_page(page->list.prev);
+
+ if ( !__page_list_del_head(page, head, next, prev) )
+ {
+ next->list.prev = page->list.prev;
+ prev->list.next = page->list.next;
+ }
+}
+static inline void
+page_list_del2(struct page_info *page, struct page_list_head *head1,
+ struct page_list_head *head2)
+{
+ struct page_info *next = mfn_to_page(page->list.next);
+ struct page_info *prev = mfn_to_page(page->list.prev);
+
+ if ( !__page_list_del_head(page, head1, next, prev) &&
+ !__page_list_del_head(page, head2, next, prev) )
+ {
+ next->list.prev = page->list.prev;
+ prev->list.next = page->list.next;
+ }
+}
+static inline void
+page_list_move_tail(struct page_info *page, struct page_list_head *list,
+ struct page_list_head *head)
+{
+ page_list_del(page, list);
+ page_list_add_tail(page, head);
+}
+static inline struct page_info *
+page_list_remove_head(struct page_list_head *head)
+{
+ struct page_info *page = head->next;
+
+ if ( page )
+ page_list_del(page, head);
+
+ return page;
+}
+static inline void
+page_list_splice_init(struct page_list_head *list, struct page_list_head *head)
+{
+ if ( !page_list_empty(list) )
+ {
+ if ( head->next )
+ head->tail->list.next = page_to_mfn(list->next);
+ else
+ head->next = list->next;
+ head->tail = list->tail;
+ INIT_PAGE_LIST_HEAD(list);
+ }
+}
+
+#define page_list_for_each(pos, head) \
+ for ( pos = (head)->next; pos; pos = page_list_next(pos, head) )
+#define page_list_for_each_safe(pos, tmp, head) \
+ for ( pos = (head)->next; \
+ pos ? (tmp = page_list_next(pos, head), 1) : 0; \
+ pos = tmp )
+#define page_list_for_each_safe_reverse(pos, tmp, head) \
+ for ( pos = (head)->tail; \
+ pos ? (tmp = page_list_prev(pos, head), 1) : 0; \
+ pos = tmp )
+#else
+# define page_list_head list_head
+# define PAGE_LIST_HEAD_INIT LIST_HEAD_INIT
+# define PAGE_LIST_HEAD LIST_HEAD
+# define INIT_PAGE_LIST_HEAD INIT_LIST_HEAD
+# define INIT_PAGE_LIST_ENTRY INIT_LIST_HEAD
+# define page_list_empty list_empty
+# define page_list_first(hd) list_entry((hd)->next, \
+ struct page_info, list)
+# define page_list_next(pg, hd) list_entry((pg)->list.next, \
+ struct page_info, list)
+# define page_list_is_eol(pg, hd) (&(pg)->list == (hd))
+# define page_list_add(pg, hd) list_add(&(pg)->list, hd)
+# define page_list_add_tail(pg, hd) list_add_tail(&(pg)->list, hd)
+# define page_list_del(pg, hd) list_del(&(pg)->list)
+# define page_list_del2(pg, hd1, hd2) list_del(&(pg)->list)
+# define page_list_move_tail(pg, o, n) list_move_tail(&(pg)->list, n)
+# define page_list_remove_head(hd) (!page_list_empty(hd) ? \
+ ({ \
+ struct page_info *__pg = page_list_first(hd); \
+ list_del(&__pg->list); \
+ __pg; \
+ }) : NULL)
+# define page_list_splice_init list_splice_init
+# define page_list_for_each(pos, head) list_for_each_entry(pos, head, list)
+# define page_list_for_each_safe(pos, tmp, head) \
+ list_for_each_entry_safe(pos, tmp, head, list)
+# define page_list_for_each_safe_reverse(pos, tmp, head) \
+ list_for_each_entry_safe_reverse(pos, tmp, head, list)
+#endif
+
/* Automatic page scrubbing for dead domains. */
-extern struct list_head page_scrub_list;
-#define page_scrub_schedule_work() \
- do { \
- if ( !list_empty(&page_scrub_list) ) \
- raise_softirq(PAGE_SCRUB_SOFTIRQ); \
+extern struct page_list_head page_scrub_list;
+#define page_scrub_schedule_work() \
+ do { \
+ if ( !page_list_empty(&page_scrub_list) ) \
+ raise_softirq(PAGE_SCRUB_SOFTIRQ); \
} while ( 0 )
#define page_scrub_kick() \
do { \
- if ( !list_empty(&page_scrub_list) ) \
+ if ( !page_list_empty(&page_scrub_list) ) \
cpumask_raise_softirq(cpu_online_map, PAGE_SCRUB_SOFTIRQ); \
} while ( 0 )
unsigned long avail_scrub_pages(void);
-#include <asm/mm.h>
-
int guest_remove_page(struct domain *d, unsigned long gmfn);
/* Returns TRUE if the whole page at @mfn is ordinary RAM. */
spinlock_t domain_lock;
spinlock_t page_alloc_lock; /* protects all the following fields */
- struct list_head page_list; /* linked list, of size tot_pages */
- struct list_head xenpage_list; /* linked list, of size xenheap_pages */
+ struct page_list_head page_list; /* linked list, of size tot_pages */
+ struct page_list_head xenpage_list; /* linked list (size xenheap_pages) */
unsigned int tot_pages; /* number of pages currently possesed */
unsigned int max_pages; /* maximum value for tot_pages */
unsigned int xenheap_pages; /* # pages allocated from Xen heap */
projects / xen.git / commitdiff